{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import os.path\n",
    "\n",
    "import get_info\n",
    "import load"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model_cnn, model_lr, model_ensemble = pickle.load(open('models.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_batch(i_batch):\n",
    "\n",
    "    # Check if there is work to do\n",
    "    cnn_features_dir = '/data/projects/opensolarmap/cnn_cache/chunks/'\n",
    "    cnn_features_filename = cnn_features_dir + 'chunk_{}.pkl'.format(i_batch)\n",
    "\n",
    "    if not os.path.isfile(cnn_features_filename):\n",
    "        print('No CNN features for batch {}'.format(i_batch))\n",
    "        return\n",
    "    \n",
    "    probas_dir = '/server/var/data/OpenSolarMap/probas/chunks/'\n",
    "    probas_filename = probas_dir + 'probas_{:04d}'.format(i_batch)\n",
    "    \n",
    "    if os.path.isfile(probas_filename):\n",
    "        #print('Probas already computed for batch {}'.format(i_batch))\n",
    "        return\n",
    "    \n",
    "    print('Processing batch {}...'.format(i_batch))\n",
    "    \n",
    "    # Load data\n",
    "    \n",
    "    building_info_dir = '/server/var/data/OpenSolarMap/buildings/chunks/'\n",
    "    building_info_filename = building_info_dir + 'building_chunk_{:04d}'.format(i_batch)\n",
    "    \n",
    "    building_info = get_info.process_building_info(building_info_filename)\n",
    "    \n",
    "    f = open(cnn_features_filename, 'rb')\n",
    "\n",
    "    try:\n",
    "        while True:\n",
    "            ident, cnn_features = pickle.load(f)\n",
    "            building_info[ident]['cnn_features'] = cnn_features\n",
    "    except EOFError:\n",
    "        pass\n",
    "\n",
    "    f.close()\n",
    "    \n",
    "    ident_list = list(building_info.keys())\n",
    "    N = len(ident_list)\n",
    "    X_cnn = np.zeros((N, 4608))\n",
    "    for i, ident in enumerate(ident_list):\n",
    "        X_cnn[i, :] = building_info[ident]['cnn_features']\n",
    "        \n",
    "        \n",
    "    l = 6\n",
    "    color = False\n",
    "    X_reduc = load.load_data(ident_list, l ,l, color)\n",
    "    \n",
    "    # Use models\n",
    "    \n",
    "    cnn_proba_test = model_cnn.predict_proba(X_cnn)[:,:-1]\n",
    "    reduc_proba_test = model_lr.predict_proba(X_reduc)[:,:-1]\n",
    "    test_mix = np.concatenate([cnn_proba_test, reduc_proba_test], axis=1)\n",
    "    \n",
    "    # predictions = model_ensemble.predict(test_mix)\n",
    "    probas = model_ensemble.predict_proba(test_mix)\n",
    "    \n",
    "    for i, ident in enumerate(ident_list):\n",
    "        building_info[ident]['probas'] = list(probas[i])\n",
    "        del building_info[ident]['cnn_features']\n",
    "        \n",
    "    # Save probas\n",
    "    pickle.dump(building_info, open(probas_filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No CNN features for batch 5\n",
      "No CNN features for batch 240\n",
      "No CNN features for batch 241\n",
      "No CNN features for batch 331\n",
      "No CNN features for batch 332\n",
      "No CNN features for batch 339\n",
      "No CNN features for batch 340\n",
      "No CNN features for batch 344\n",
      "No CNN features for batch 351\n",
      "No CNN features for batch 400\n",
      "No CNN features for batch 459\n",
      "No CNN features for batch 476\n",
      "No CNN features for batch 484\n",
      "No CNN features for batch 485\n",
      "No CNN features for batch 492\n",
      "No CNN features for batch 521\n",
      "No CNN features for batch 560\n",
      "No CNN features for batch 584\n",
      "No CNN features for batch 610\n",
      "No CNN features for batch 702\n",
      "No CNN features for batch 801\n",
      "No CNN features for batch 908\n",
      "No CNN features for batch 951\n",
      "No CNN features for batch 964\n",
      "No CNN features for batch 971\n",
      "No CNN features for batch 1019\n",
      "No CNN features for batch 1025\n",
      "No CNN features for batch 1166\n",
      "No CNN features for batch 1167\n",
      "No CNN features for batch 1168\n",
      "No CNN features for batch 1267\n",
      "No CNN features for batch 1296\n",
      "No CNN features for batch 1384\n",
      "No CNN features for batch 1385\n",
      "No CNN features for batch 1386\n",
      "No CNN features for batch 1387\n",
      "No CNN features for batch 1388\n"
     ]
    }
   ],
   "source": [
    "for i_batch in range(1410):\n",
    "    process_batch(i_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
